# -*- coding:utf-8 -*-
import os
from hashlib import md5
import re
import json
import time
import requests
import pymongo
from config import *


headers = {
	'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36',
}


def get_detail_urls(offset, keyword):
	data = {
		'offset': offset,
		'format': 'json',
		'keyword': keyword,
		'autoload': 'true',
		'count': '20',
		'cur_tab': 1
	}

	start_url = 'https://www.toutiao.com/search_content/'
	try:
		req_data = requests.get(start_url, headers=headers, params=data)
		if req_data.status_code == 200:
			# data = json.loads(req_data.text)	# 使用原生json将json对象转化为python对象
			get_data = req_data.json()
			if get_data and 'data' in get_data.keys():
				for item in get_data.get('data'):
					yield item.get('article_url')
		yield None
	except requests.RequestException:
		print '请求出错', start_url
		yield None


def get_detail_info(detail_url):
	try:
		req_data = requests.get(detail_url, headers=headers)
		if req_data.status_code == 200:
			get_data = req_data.text
			if get_data:
				# 正则取出标题、图片url
				title_pattern = re.compile('title:(.*?),', re.S)
				content_pattern = re.compile('content:(.*?),', re.S)
				img_pattern = re.compile(';http(.*?)&quot;', re.S)
				title = re.findall(title_pattern, get_data)[0]
				content = re.findall(content_pattern, get_data)
				if content:
					content = content[0]
				else:
					content = ''
				img = re.findall(img_pattern,content)
				img_url = ['http' + i for i in img]
				for i in img_url:
					download_detail_img(i)
				# 封装到字典里
				info_data_dict = {
					'title': title,
					'img_url': img_url,
				}
				return info_data_dict
		return None
	except requests.RequestException:
		print '请求出错', detail_url
		return None


def save_detail_info(detail_info):
	client = pymongo.MongoClient(HOST,PORT,username=USERNAME,password=PASSWORD)
	db = client.admin
	tb = db.toutiaoJiepai
	if detail_info:
		tb.insert(detail_info)
	else:
		pass


def save_detail_img(img):
	img_path = '{0}/{1}{2}'.format('JiepaiImg', md5(img).hexdigest(), '.png')
	if not os.path.exists(img_path):
		with open(img_path, 'wb') as f:
			f.write(img)
			f.close()


def download_detail_img(img_url):
	print '正在下载：{}'.format(img_url)
	try:
		req_data = requests.get(img_url)
		if req_data.status_code == 200:
			save_detail_img(req_data.content)
	except requests.RequestException:
		print '请求出错', img_url
		return None



def main():
	detail_urls = get_detail_urls(0,'街拍')
	for detail_url in detail_urls:
	# 	print detail_url
		detail_info = get_detail_info(detail_url)
		# print detail_info
		save_detail_info(detail_info)
		time.sleep(2)


if __name__ == '__main__':
	main()
